# coding=utf-8
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import time
import logging
import random
import re
from selenium.webdriver import ActionChains
import urllib
import ssl
from fake_useragent import UserAgent
import datetime
import json
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import os
from toutiao_read_excel import read_excel
from insert_sql import insert_etl_db
import subprocess

ssl._create_default_https_context = ssl._create_unverified_context
ua = UserAgent(verify_ssl=False)
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S')


class crawl_toutiao:

    def __init__(self):

        # 头条登录页面
        self.login_url = "https://business.oceanengine.com/site/login"
        self.last_day = (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y-%m-%d')
        self.dowload_file = self.last_day + "_file.xlsx"
        self.toutiao_download_file_dir = "toutiao_download_file_dir/" + self.last_day
        self.name = "name"
        self.password = "pwd"
        # self.driver = webdriver.PhantomJS(executable_path=r'/home/postgres/haoyin/phantomjs_dir/phantomjs-2.1.1-linux-x86_64/bin/phantomjs')
        self.chrome_options = webdriver.ChromeOptions()
        self.chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.chrome_options.add_argument('--headless')
        # self.chrome_options.add_experimental_option('excludeSwitches', ['enable-debug'])
        # self.chrome_options.add_argument('log-level=3')
        self.chrome_options.add_argument(ua.chrome)
        # self.chrome_options.add_argument('--proxy-server=http://171.37.135.94:8123')
        self.driver = webdriver.Chrome(executable_path="/bin/chromedriver",
                                       chrome_options=self.chrome_options)
        self.toutiao_pic_dir = "./toutiao_pic_dir/"
        # self.expense_forms_url = "https://e.oceanengine.com/account/page/fund/flow/income_expanditure?cc_account_id=2717594251169420"
        self.expense_forms_url = "https://business.oceanengine.com/site/finance?cc_id=2717594251169420&type=income_expanditure"

    def login(self):

        subprocess.call("rm -f 度小满保险*.xlsx", shell=True)
        self.driver.get(self.login_url)
        time.sleep(round(random.uniform(0.71, 1.59), 1))
        # 邮箱
        for single_letter in self.name:
            WebDriverWait(self.driver, 5).until(
                lambda driver: self.driver.find_element_by_xpath(
                    '//*[@id="login"]/section/div[3]/div[1]/div[2]/div/input')).send_keys("%s" % single_letter)
            time.sleep(round(random.uniform(0.11, 0.39), 1))
        time.sleep(round(random.uniform(0.11, 0.29), 1))
        # 密码
        for single_letter in self.password:
            WebDriverWait(self.driver, 5).until(
                lambda driver: self.driver.find_element_by_xpath(
                    '//*[@id="login"]/section/div[3]/div[2]/div/div/input')).send_keys("%s" % single_letter)
            time.sleep(round(random.uniform(0.11, 0.29), 1))
        time.sleep(round(random.uniform(0.11, 0.69), 1))
        # 登陆
        WebDriverWait(self.driver, 5).until(
            lambda driver: self.driver.find_element_by_xpath(
                '//*[@id="login"]/section/div[6]/button')).click()
        time.sleep(round(random.uniform(0.51, 1.39), 1))

    def exec_login_crawl(self):

        try:
            self.login()
        except Exception as e:
            print(e)
        try:
            time.sleep(round(random.uniform(5.71, 7.59), 1))
            self.offical_crawl_expense()
        finally:
            self.driver.quit()

    def judge_dir(self, need_check_dir):

        # 判断前一天日志存储目录是否存在，不存在创建
        if not os.path.exists(need_check_dir):
            os.makedirs(need_check_dir)

        else:
            logging.info(need_check_dir + " already exists !")
            subprocess.call("rm -f {data}/度小满保险*.xlsx".format(data=need_check_dir), shell=True)

    def download_xlsx(self):
        retry_num = 1
        # download button
        WebDriverWait(self.driver, 5).until(
            lambda driver: self.driver.find_element_by_xpath(
                '//*[@id="app"]/div/div/div[2]/div/div[1]/div[2]/button')).click()
        time.sleep(round(random.uniform(4.71, 5.59), 1))

        while True:
            if re.findall(r",(度小满保险.*xlsx)", ",".join(os.listdir())):
                self.judge_dir(self.toutiao_download_file_dir)
                subprocess.call("mv -f 度小满保险*.xlsx {}/{}".format(self.toutiao_download_file_dir, self.dowload_file),
                                shell=True)
                cost = read_excel(self.toutiao_download_file_dir + "/" + self.dowload_file).exec_main()
                print("总额", cost)
                if cost:
                    insert_etl_db(str(cost), 'toufang_toutiao', 'duocai').insert_mysql()
                    break
                retry_num += 1
                if retry_num == 5:
                    break
                break
            else:
                time.sleep(round(random.uniform(3.71, 4.59), 1))

    def offical_crawl_expense(self):

        retry_num = 1
        while True:
            self.driver.get(self.expense_forms_url)
            time.sleep(round(random.uniform(3.71, 4.59), 1))
            if self.driver.current_url != self.expense_forms_url:
                time.sleep(round(random.uniform(3.71, 4.59), 1))
                self.login()
                retry_num += 1
                if retry_num == 5:
                    break
            else:
                break
        time.sleep(round(random.uniform(2.71, 3.59), 1))
        # 下载表格
        self.driver.switch_to.frame(self.driver.find_element_by_xpath('/html/body/div[2]/iframe'))
        time.sleep(round(random.uniform(0.71, 1.59), 1))
        self.driver.switch_to.frame(self.driver.find_element_by_xpath('//*[@id="fund"]/div/iframe'))
        time.sleep(round(random.uniform(0.71, 1.59), 1))

        if (datetime.datetime.now()).strftime('%d') == "01":
            # calendar
            WebDriverWait(self.driver, 5).until(
                lambda driver: self.driver.find_element_by_xpath(
                    '//*[@id="app"]/div/div/div[1]/div[1]/div[4]/div/div/div/input')).click()
            time.sleep(round(random.uniform(1.71, 2.59), 1))
            # 点击日期
            td = int((datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%w'))+1
            WebDriverWait(self.driver, 5).until(
                lambda driver: self.driver.find_element_by_xpath(
                    '/html/body/div[2]/div/div[1]/div[2]/table/tbody/tr[1]/td[%d]/span' % td)).click()
            time.sleep(round(random.uniform(0.71, 1.59), 1))
            WebDriverWait(self.driver, 5).until(
                lambda driver: self.driver.find_element_by_xpath(
                    '/html/body/div[2]/div/div[1]/div[2]/table/tbody/tr[1]/td[%d]/span' % td)).click()
            time.sleep(round(random.uniform(1.71, 2.59), 1))
            ori_size = self.driver.get_window_size()
            self.driver.set_window_size(1500, ori_size['height'])
            time.sleep(round(random.uniform(1.71, 2.59), 1))
            # search button
            WebDriverWait(self.driver, 5).until(
                lambda driver: self.driver.find_element_by_xpath(
                    '//*[@id="app"]/div/div/div[1]/div[2]/button[1]')).click()
            time.sleep(round(random.uniform(1.71, 2.59), 1))
            self.download_xlsx()
        else:
            self.download_xlsx()


if __name__ == '__main__':
    ct = crawl_toutiao()
    ct.exec_login_crawl()
# with ProcessPoolExecutor(4) as p:
# p.submit(crawl_toutiao().exec_login_crawl)
# p.submit(crawl_baidu().exec_login_crawl)
